From 8550e20c7fb4eff012ccfc7126f5717ce92c8f0a Mon Sep 17 00:00:00 2001 From: "kaf24@labyrinth.cl.cam.ac.uk" Date: Tue, 24 Aug 2004 15:12:02 +0000 Subject: [PATCH] bitkeeper revision 1.1159.1.101 (412b5ac2PQ9FDoJKc14Km1yEm114Rw) Grant-table pin/unpin operation. --- xen/common/grant_table.c | 198 +++++++++++++++++++---- xen/include/asm-x86/system.h | 62 ++++--- xen/include/hypervisor-ifs/grant_table.h | 13 +- 3 files changed, 218 insertions(+), 55 deletions(-) diff --git a/xen/common/grant_table.c b/xen/common/grant_table.c index 68502a6925..27f81b9e22 100644 --- a/xen/common/grant_table.c +++ b/xen/common/grant_table.c @@ -24,25 +24,63 @@ #include #include -#define update_shared_flags(x,y,z) (0) +static inline void +check_tlb_flush( + active_grant_entry_t *a) +{ + if ( unlikely(NEED_FLUSH(tlbflush_time[smp_processor_id()], + a->tlbflush_timestamp)) ) + { + perfc_incr(need_flush_tlb_flush); + local_flush_tlb(); + } +} -static long gnttab_update_pin_status(gnttab_update_pin_status_t *uop) +static void +make_entry_mappable( + grant_table_t *t, active_grant_entry_t *a) +{ + u16 *ph = &t->maphash[GNT_MAPHASH(a->frame)]; + a->next = *ph; + *ph = a - t->active; +} + +static void +make_entry_unmappable( + grant_table_t *t, active_grant_entry_t *a) +{ + active_grant_entry_t *p; + u16 *ph = &t->maphash[GNT_MAPHASH(a->frame)]; + while ( (p = &t->active[*ph]) != a ) + ph = &p->next; + *ph = a->next; + a->next = GNT_MAPHASH_INVALID; + check_tlb_flush(a); +} + +static long +gnttab_update_pin_status( + gnttab_update_pin_status_t *uop) { domid_t dom, sdom; grant_ref_t ref; u16 pin_flags; struct domain *ld, *rd; - u32 sflags; + u16 sflags, prev_sflags; active_grant_entry_t *act; grant_entry_t *sha; long rc = 0; ld = current; - if ( unlikely(__get_user(dom, &uop->dom)) || - unlikely(__get_user(ref, &uop->ref)) || - unlikely(__get_user(pin_flags, &uop->pin_flags)) ) + /* Bitwise-OR avoids short-circuiting which screws control flow. */ + if ( unlikely(__get_user(dom, &uop->dom) | + __get_user(ref, &uop->ref) | + __get_user(pin_flags, &uop->pin_flags)) ) + { + DPRINTK("Fault while reading gnttab_update_pin_status_t.\n"); return -EFAULT; + } pin_flags &= (GNTPIN_dev_accessible | GNTPIN_host_accessible | @@ -50,10 +88,16 @@ static long gnttab_update_pin_status(gnttab_update_pin_status_t *uop) if ( unlikely(ref >= NR_GRANT_ENTRIES) || unlikely(pin_flags == GNTPIN_readonly) ) + { + DPRINTK("Bad ref (%d) or flags (%x).\n", ref, pin_flags); return -EINVAL; + } if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ) + { + DPRINTK("Could not find domain %d\n", dom); return -ESRCH; + } act = &rd->grant_table->active[ref]; sha = &rd->grant_table->shared[ref]; @@ -63,79 +107,167 @@ static long gnttab_update_pin_status(gnttab_update_pin_status_t *uop) if ( unlikely(pin_flags == 0) ) goto out; + /* CASE 1: Activating a previously inactive entry. */ + sflags = sha->flags; sdom = sha->domid; - do { + for ( ; ; ) + { + u32 scombo, prev_scombo; + if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access) || unlikely(sdom != ld->domain) ) { + DPRINTK("Bad flags (%x) or dom (%d). (NB. expected dom %d)\n", + sflags, sdom, ld->domain); + rc = -EINVAL; + goto out; } - + sflags |= GTF_reading; if ( !(pin_flags & GNTPIN_readonly) ) { sflags |= GTF_writing; if ( unlikely(sflags & GTF_readonly) ) { + DPRINTK("Attempt to write-pin a read-only grant entry.\n"); + rc = -EINVAL; + goto out; } } + + /* Merge two 16-bit values into a 32-bit combined update. */ + /* NB. Endianness! */ + prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags; + + /* NB. prev_sflags is updated in place to seen value. */ + if ( unlikely(cmpxchg_user((u32 *)&sha->flags, prev_scombo, + prev_scombo | GTF_writing)) ) + { + DPRINTK("Fault while modifying shared flags and domid.\n"); + rc = -EINVAL; + goto out; + } + + /* Did the combined update work (did we see what we expected?). */ + if ( prev_scombo == scombo ) + break; + + /* Didn't see what we expected. Split out the seen flags & dom. */ + /* NB. Endianness! */ + sflags = (u16)prev_scombo; + sdom = (u16)(prev_scombo >> 16); } - while ( !update_shared_flags(sha, sflags, sdom) ); + + /* rmb(); */ /* not on x86 */ act->status = pin_flags; act->domid = sdom; + act->frame = sha->frame; - /* XXX MAP XXX */ + make_entry_mappable(rd->grant_table, act); } else if ( pin_flags == 0 ) { + /* CASE 2: Deactivating a previously active entry. */ + if ( unlikely((act->status & (GNTPIN_wmap_mask|GNTPIN_rmap_mask)) != 0) ) { + DPRINTK("Attempt to deactivate a mapped g.e. (%x)\n", act->status); + rc = -EINVAL; + goto out; } - clear_bit(_GTF_writing, &sha->flags); - clear_bit(_GTF_reading, &sha->flags); - act->status = 0; + make_entry_unmappable(rd->grant_table, act); - /* XXX UNMAP XXX */ + clear_bit(_GTF_writing, &sha->flags); + clear_bit(_GTF_reading, &sha->flags); } else { - if ( pin_flags & GNTPIN_readonly ) - { - if ( !(act->status & GNTPIN_readonly) ) - { - } - } - else if ( act->status & GNTPIN_readonly ) + /* CASE 3: Active modications to an already active entry. */ + + /* + * Check mapping counts up front, as necessary. + * After this compound check, the operation cannot fail. + */ + if ( ((pin_flags & (GNTPIN_readonly|GNTPIN_host_accessible)) != + GNTPIN_host_accessible) && + (unlikely((act->status & GNTPIN_wmap_mask) != 0) || + (((pin_flags & GNTPIN_host_accessible) == 0) && + unlikely((act->status & GNTPIN_rmap_mask) != 0))) ) { + DPRINTK("Attempt to reduce pinning of a mapped g.e. (%x,%x)\n", + pin_flags, act->status); + rc = -EINVAL; + goto out; } + /* Check for changes to host accessibility. */ if ( pin_flags & GNTPIN_host_accessible ) { if ( !(act->status & GNTPIN_host_accessible) ) + make_entry_mappable(rd->grant_table, act); + } + else if ( act->status & GNTPIN_host_accessible ) + make_entry_unmappable(rd->grant_table, act); + + /* Check for changes to write accessibility. */ + if ( pin_flags & GNTPIN_readonly ) + { + if ( !(act->status & GNTPIN_readonly) ) { - /* XXX MAP XXX */ + check_tlb_flush(act); + clear_bit(_GTF_writing, &sha->flags); } } - else if ( act->status & GNTPIN_host_accessible ) + else if ( act->status & GNTPIN_readonly ) { - /* XXX UNMAP XXX */ + sflags = sha->flags; + do { + prev_sflags = sflags; + + if ( unlikely(prev_sflags & GTF_readonly) ) + { + DPRINTK("Attempt to write-pin a read-only grant entry.\n"); + rc = -EINVAL; + goto out; + } + + /* NB. prev_sflags is updated in place to seen value. */ + if ( unlikely(cmpxchg_user(&sha->flags, prev_sflags, + prev_sflags | GTF_writing)) ) + { + DPRINTK("Fault while modifying shared flags.\n"); + rc = -EINVAL; + goto out; + } + } + while ( prev_sflags != sflags ); } - act->status &= ~GNTPIN_dev_accessible; - act->status |= pin_flags & GNTPIN_dev_accessible; + /* Update status word -- this includes device accessibility. */ + act->status &= ~(GNTPIN_dev_accessible | + GNTPIN_host_accessible | + GNTPIN_readonly); + act->status |= pin_flags; } + /* Unchecked and unconditional. */ + (void)__put_user(act->frame, &uop->dev_bus_addr); + (void)__put_user(act->frame, &uop->host_phys_addr); + out: put_domain(rd); return rc; } -long do_grant_table_op(gnttab_op_t *uop) +long +do_grant_table_op( + gnttab_op_t *uop) { long rc; u32 cmd; @@ -157,7 +289,9 @@ long do_grant_table_op(gnttab_op_t *uop) return rc; } -int grant_table_create(struct domain *d) +int +grant_table_create( + struct domain *d) { grant_table_t *t; int i; @@ -197,7 +331,9 @@ int grant_table_create(struct domain *d) return -ENOMEM; } -void grant_table_destroy(struct domain *d) +void +grant_table_destroy( + struct domain *d) { grant_table_t *t; @@ -211,7 +347,9 @@ void grant_table_destroy(struct domain *d) } } -void grant_table_init(void) +void +grant_table_init( + void) { /* Nothing. */ } diff --git a/xen/include/asm-x86/system.h b/xen/include/asm-x86/system.h index ff88ed4692..4b25eec921 100644 --- a/xen/include/asm-x86/system.h +++ b/xen/include/asm-x86/system.h @@ -11,14 +11,6 @@ #define wbinvd() \ __asm__ __volatile__ ("wbinvd": : :"memory"); -static inline unsigned long get_limit(unsigned long segment) -{ - unsigned long __limit; - __asm__("lsll %1,%0" - :"=r" (__limit):"r" (segment)); - return __limit+1; -} - #define nop() __asm__ __volatile__ ("nop") #define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr)))) @@ -32,7 +24,7 @@ struct __xchg_dummy { unsigned long a[100]; }; * Note 2: xchg has side effect, so that attribute volatile is necessary, * but generally the primitive is invalid, *ptr is output argument. --ANK */ -static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) +static always_inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) { switch (size) { case 1: @@ -78,7 +70,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int siz * indicated by comparing RETURN with OLD. */ -static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, +static always_inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) { unsigned long prev; @@ -126,17 +118,14 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, /* - * This function causes longword _o to be changed to _n at location _p. + * This function causes value _o to be changed to _n at location _p. * If this access causes a fault then we return 1, otherwise we return 0. - * If no fault occurs then _o is updated to teh value we saw at _p. If this + * If no fault occurs then _o is updated to the value we saw at _p. If this * is the same as the initial value of _o then _n is written to location _p. */ -#ifdef __i386__ -#define cmpxchg_user(_p,_o,_n) \ -({ \ - int _rc; \ +#define __cmpxchg_user(_p,_o,_n,_isuff,_oppre,_regtype) \ __asm__ __volatile__ ( \ - "1: " LOCK_PREFIX "cmpxchg"__OS" %2,%3\n" \ + "1: " LOCK_PREFIX "cmpxchg"_isuff" %"_oppre"2,%3\n" \ "2:\n" \ ".section .fixup,\"ax\"\n" \ "3: movl $1,%1\n" \ @@ -147,12 +136,45 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, " .long 1b,3b\n" \ ".previous" \ : "=a" (_o), "=r" (_rc) \ - : "q" (_n), "m" (*__xg((volatile void *)_p)), "0" (_o), "1" (0) \ - : "memory"); \ + : _regtype (_n), "m" (*__xg((volatile void *)_p)), "0" (_o), "1" (0) \ + : "memory"); +#ifdef __i386__ +#define cmpxchg_user(_p,_o,_n) \ +({ \ + int _rc; \ + switch ( sizeof(*(_p)) ) { \ + case 1: \ + __cmpxchg_user(_p,_o,_n,"b","b","q"); \ + break; \ + case 2: \ + __cmpxchg_user(_p,_o,_n,"w","w","r"); \ + break; \ + case 4: \ + __cmpxchg_user(_p,_o,_n,"l","","r"); \ + break; \ + } \ _rc; \ }) #else -#define cmpxchg_user(_p,_o,_n) ({ __asm__ __volatile__ ( "" : : "r" (_p), "r" (_o), "r" (_n) ); BUG(); 0; }) +#define cmpxchg_user(_p,_o,_n) \ +({ \ + int _rc; \ + switch ( sizeof(*(_p)) ) { \ + case 1: \ + __cmpxchg_user(_p,_o,_n,"b","b","q"); \ + break; \ + case 2: \ + __cmpxchg_user(_p,_o,_n,"w","w","r"); \ + break; \ + case 4: \ + __cmpxchg_user(_p,_o,_n,"l","k","r"); \ + break; \ + case 8: \ + __cmpxchg_user(_p,_o,_n,"q","","r"); \ + break; \ + } \ + _rc; \ +}) #endif /* diff --git a/xen/include/hypervisor-ifs/grant_table.h b/xen/include/hypervisor-ifs/grant_table.h index dcfa8859c2..97513cce60 100644 --- a/xen/include/hypervisor-ifs/grant_table.h +++ b/xen/include/hypervisor-ifs/grant_table.h @@ -16,10 +16,11 @@ */ /* Some rough guidelines on accessing and updating grant-table entries - * in a concurreny-safe manner. For more information, Linux contains a + * in a concurrency-safe manner. For more information, Linux contains a * reference implementation for guest OSes (arch/xen/kernel/grant_table.c). * - * NB. WMB is a no-op on current-generation x86 processors. + * NB. WMB is a no-op on current-generation x86 processors. However, a + * compiler barrier will still be required. * * Introducing a valid entry into the grant table: * 1. Write ent->domid. @@ -31,11 +32,13 @@ * 1. flags = ent->flags. * 2. Observe that !(flags & (GTF_reading|GTF_writing)). * 3. Check result of SMP-safe CMPXCHG(&ent->flags, flags, 0). - * 4. WMB. + * NB. No need for WMB as reuse of entry is control-dependent on success of + * step 3, and all architectures guarantee ordering of ctrl-dep writes. * * Removing an unused GTF_accept_transfer entry: - * 1. Clear ent->flags. - * 2. WMB. + * 1. Check result of SMP-safe CMPXCHG(&ent->frame, 0, ). + * 2. Clear ent->flags. + * 3. WMB (ordering of step 2 vs. steps 1,2 of introducing a new entry). * * Changing a GTF_permit_access from writable to read-only: * Use SMP-safe CMPXCHG to set GTF_readonly, while checking !GTF_writing. -- 2.30.2